from pyspark.sql.session import SparkSession
from pyspark.context import SparkContext
from pyspark import StorageLevel

from pyspark.conf import SparkConf

spark = SparkSession.builder.config(conf=SparkConf()).enableHiveSupport().getOrCreate()
sc =  spark.sparkContext

log4jLogger = sc._jvm.org.apache.log4j
LOGGER = log4jLogger.LogManager.getLogger(__name__)

import sys
import pathlib
import argparse

def parse_argv():
    parser = argparse.ArgumentParser()
    parser.add_argument('--conf', metavar='CONF_DIR', default=default_configure_dir, help='Specify configure file')
    return parser.parse_args(sys.argv[1:])

project_root = pathlib.Path(__file__).parent.parent
default_configure_dir = project_root.joinpath('conf').absolute()
args = parse_argv()

pathlib.Path('.').joinpath('logs').mkdir(exist_ok=True)
spark.sql("create database  IF NOT EXISTS profile")